{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# feature engineering\n",
    "特征工程是利用数据领域的相关知识来创建能够使机器学习算法达到最佳性能的特征的过程。简而言之，特征工程就是一个把原始数据转变成特征的过程，这些特征可以很好的描述这些数据，并且利用它们建立的模型在未知数据上的表现性能可以达到最优（或者接近最佳性能）。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import lightgbm as lgb\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parseData(df):\n",
    "    \"\"\"\n",
    "    预处理数据\n",
    "    \"\"\"\n",
    "    df['rentType'][df['rentType']=='--'] = '未知方式'\n",
    "    \n",
    "    # object类型数据转换为category\n",
    "    df['region'] = [re[5: ] for re in df['region']]\n",
    "    df['plate'] = [pl[5: ] for pl in df['plate']]\n",
    "    columns = ['rentType', 'houseFloor', 'houseToward', 'houseDecoration', 'communityName', 'region', 'plate']\n",
    "    for col in columns:\n",
    "        df[col] = df[col].astype('category')\n",
    "        \n",
    "    # buildYear列转换为整型数据，并将缺失值填充为众数\n",
    "    tmp = df['buildYear'].copy()\n",
    "    tmp2 = tmp[tmp!='暂无信息'].astype('int')\n",
    "    tmp[tmp=='暂无信息'] = tmp2.mode().iloc[0]\n",
    "    df['buildYear'] = tmp\n",
    "    df['buildYear'] = df['buildYear'].astype('int')\n",
    "    \n",
    "    # pv和uv的缺失值填充为均值，并且转换为整型\n",
    "    df['pv'].fillna(df['pv'].mean(),inplace=True)\n",
    "    df['uv'].fillna(df['uv'].mean(),inplace=True)\n",
    "    df['pv'] = df['pv'].astype('int')\n",
    "    df['uv'] = df['uv'].astype('int')\n",
    "    \n",
    "    # 去掉部分特征\n",
    "    df.drop('city',axis=1,inplace=True)\n",
    "    \n",
    "    \n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def washData(df_train, df_test):\n",
    "    \"\"\"\n",
    "    清洗数据\n",
    "    \"\"\"\n",
    "    # 根据异常值检测得出区间\n",
    "    df_train = df_train[df_train['area']<=200]\n",
    "    df_train = df_train[df_train['tradeMoney']<=10000]\n",
    "    df_train = df_train[df_train['totalFloor']<=50]\n",
    "    df_train = df_train[df_train['saleSecHouseNum']<=5]\n",
    "    df_train = df_train[df_train['remainNewNum']<=1500]\n",
    "    \n",
    "    df_train.drop('ID', axis=1, inplace=True)\n",
    "    df_test.drop('ID', axis=1,inplace=True)\n",
    "    \n",
    "    def rentType_trans(rt):\n",
    "        if rt == '未知方式':\n",
    "            return 0\n",
    "        elif rt == '整租':\n",
    "            return 1\n",
    "        elif rt == '合租':\n",
    "            return 2\n",
    "        else:\n",
    "            return 0\n",
    "\n",
    "    df['rentType'] = df['rentType'].apply(rentType_trans)\n",
    "    \n",
    "    # 楼层高低数值化    \n",
    "    def houseFloor_trans(hf):\n",
    "        if hf == '低':\n",
    "            return 0\n",
    "        elif hf == '中':\n",
    "            return 1\n",
    "        else:\n",
    "            return 2\n",
    "\n",
    "    df['houseFloor'] = df['houseFloor'].apply(houseFloor_trans)\n",
    "\n",
    "    \n",
    "    # 房屋朝向数值化\n",
    "    def houseToward_trans(ht):\n",
    "        if ht in ['南', '南北', '东南', '西南']:\n",
    "            return 1\n",
    "        elif ht in ['东', '东西', '西']:\n",
    "            return 2\n",
    "        elif ht in ['西北','东北']:\n",
    "            return 3\n",
    "        else:\n",
    "            return 4\n",
    "\n",
    "    df['houseToward'] = df['houseToward'].apply(houseToward_trans)\n",
    "\n",
    "    # 装修类型\n",
    "    def houseDecoration_trans(hd):\n",
    "        if hd == '毛坯':\n",
    "            return 1\n",
    "        elif hd == '简装':\n",
    "            return 2\n",
    "        else:\n",
    "            return 3\n",
    "\n",
    "    df['houseDecoration'] = df['houseDecoration'].apply(houseDecoration_trans)\n",
    "    return df_train, df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature(df):\n",
    "    \"\"\"\n",
    "    特征\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    # 根据房型信息抽取出更细的信息\n",
    "    df['room'] = [int(ht[0]) for ht in df['houseType']]\n",
    "    df['hall'] = [int(ht[2]) for ht in df['houseType']]\n",
    "    df['bath'] = [int(ht[-2]) for ht in df['houseType']]\n",
    "    df['totalRoom'] = df['room'] + df['hall'] + df['bath']\n",
    "    df['area_per_room'] = df['area'] / df['totalRoom']\n",
    "    \n",
    "    \n",
    "    # rentType缺失值的填充\n",
    "    df.loc[(df['area'] <= 50) & (df['room'] >= 3) & (df['rentType'] == 0), 'rentType'] = 2\n",
    "    df.loc[(df['rentType'] == 2) & (df['area'] > 50), 'area'] = df['area'] / (df['room'] + 1)\n",
    "\n",
    "\n",
    "    df.loc[(df['houseDecoration'] == 1) & (df['rentType'] == 0), 'rentType'] = 1\n",
    "\n",
    "    \n",
    "    df['communityName'] = [int(cn[2: ]) for cn in df['communityName']]\n",
    "    \n",
    "    \n",
    "    # 交易月份\n",
    "    df['trade_month'] = [int(time.split('/')[1]) for time in df['tradeTime']]\n",
    "    # 交易季节\n",
    "    df['season'] = [int(np.ceil(month / 3)) for month in df['trade_month']]\n",
    "\n",
    "    # 大致所在楼层\n",
    "    df['floor_ratio'] = round(df['totalFloor'] * ((df['houseFloor'] * 2 + 1) / 6))\n",
    "    df['per_pv'] = df['pv'] / df['uv']\n",
    "    df['mean_pv'] = (df['pv'] + df['uv']) / 2\n",
    "    df['max_pv'] = np.max(df[['pv', 'uv']], axis=1)\n",
    "    df['min_pv'] = np.min(df[['pv', 'uv']], axis=1)\n",
    "    df['std_pv'] = np.std(df[['pv', 'uv']], axis=1)\n",
    "    df['worker_ratio'] = df['totalWorkers'] / df['residentPopulation']\n",
    "    \n",
    "    df['room'] = [1 if rt == 2 else room for rt, room in np.array(df[['rentType', 'room']])]\n",
    "    df['totalRoom'] = [1 if rt == 2 else tr for rt, tr in np.array(df[['rentType', 'totalRoom']])]\n",
    "    \n",
    "    a = df.groupby(['plate', 'rentType']).mean().reset_index()[['plate', 'rentType', 'tradeMoney']].sort_values('tradeMoney', ascending=False).rename(columns={'tradeMoney': 'plate_mean'})\n",
    "    b = df.groupby(['region', 'rentType']).mean().reset_index()[['region', 'rentType', 'tradeMoney']].sort_values('tradeMoney', ascending=False).rename(columns={'tradeMoney': 'region_mean'})\n",
    "    c = df.groupby(['plate', 'rentType']).median().reset_index()[['plate', 'rentType', 'tradeMoney']].sort_values('tradeMoney', ascending=False).rename(columns={'tradeMoney': 'plate_median'})\n",
    "    d = df.groupby(['region', 'rentType']).median().reset_index()[['region', 'rentType', 'tradeMoney']].sort_values('tradeMoney', ascending=False).rename(columns={'tradeMoney': 'region_median'})\n",
    "\n",
    "    df = df.merge(a, how='left', on=['plate', 'rentType'])\n",
    "    df = df.merge(b, how='left', on=['region', 'rentType'])\n",
    "    df = df.merge(c, how='left', on=['plate', 'rentType'])\n",
    "    df = df.merge(d, how='left', on=['region', 'rentType'])\n",
    "    \n",
    "    a = df.groupby(['region']).size().reset_index().rename(columns={0: 'region_num'})\n",
    "    df = df.merge(a, how='left', on=['region'])\n",
    "    a = df.groupby(['plate']).size().reset_index().rename(columns={0: 'plate_num'})\n",
    "    df = df.merge(a, how='left', on=['plate'])\n",
    "    a = df.groupby(['communityName']).size().reset_index().rename(columns={0: 'community_num'})\n",
    "    df = df.merge(a, how='left', on=['communityName'])\n",
    "    a = df[['region', 'plate']].groupby(['region']).apply(lambda x: x['plate'].nunique()).reset_index().rename(columns={0: 'plate_contain'})\n",
    "    df = df.merge(a, how='left', on=['region'])\n",
    "    a = df.groupby('plate').apply(lambda x: x['communityName'].nunique()).reset_index().rename(columns={0: 'community_contain'})\n",
    "    df = df.merge(a, how='left', on=['plate'])\n",
    "    \n",
    "    a = df.groupby('plate').mean().reset_index()[['plate', 'totalFloor']].rename(columns={'totalFloor': 'plate_mean_floor'})\n",
    "    df = df.merge(a, how='left', on=['plate'])\n",
    "    a = df.groupby('plate').median().reset_index()[['plate', 'totalFloor']].rename(columns={'totalFloor': 'plate_median_floor'})\n",
    "    df = df.merge(a, how='left', on=['plate'])\n",
    "    a = df.groupby('plate').mean().reset_index()[['plate', 'area']].rename(columns={'area': 'plate_mean_area'})\n",
    "    df = df.merge(a, how='left', on=['plate'])\n",
    "    a = df.groupby('plate').median().reset_index()[['plate', 'area']].rename(columns={'area': 'plate_median_area'})\n",
    "    df = df.merge(a, how='left', on=['plate'])\n",
    "    \n",
    "    df['room_ratio'] = df['room'] / df['totalRoom']\n",
    "    df['hall_ratio'] = df['hall'] / df['totalRoom']\n",
    "    df['bath_ratio'] = df['bath'] / df['totalRoom']\n",
    "    df['room-bath'] = df['room'] - df['bath']\n",
    "\n",
    "    df['max_type'] = np.argmax(np.array(df[['room', 'hall', 'bath']]), axis=1)\n",
    "\n",
    "    df['trade_avg'] = df['totalTradeArea'] / (df['tradeSecNum'] + 1)\n",
    "\n",
    "    df['originWorkers'] = df['totalWorkers'] - df['newWorkers']\n",
    "\n",
    "    # df['feature2'] = df['plate_mean'] / df['region_mean']\n",
    "\n",
    "    df['area_part'] = pd.qcut(df['area'], q=6, labels=[0, 1, 2, 3, 4, 5])\n",
    "\n",
    "    df['tf_part'] = pd.qcut(df['totalFloor'], q=8, duplicates='drop', labels=[i for i in range(6)])\n",
    "    \n",
    "    # plate, room, area, totalRoom\n",
    "\n",
    "    a = df.groupby(['communityName']).apply(lambda x: x['totalRoom'].tolist()).reset_index()\n",
    "    a['mean_totalRoom'] = [np.mean(i) for i in a[0]]\n",
    "    a['median_totalRoom'] = [np.median(i) for i in a[0]]\n",
    "    a = a.drop(0, axis=1)\n",
    "    df = df.merge(a, how='left', on=['communityName'])\n",
    "\n",
    "    a = df.groupby(['communityName']).apply(lambda x: x['room'].tolist()).reset_index()\n",
    "    a['mean_room'] = [np.mean(i) for i in a[0]]\n",
    "    a['median_room'] = [np.median(i) for i in a[0]]\n",
    "    a = a.drop(0, axis=1)\n",
    "    df = df.merge(a, how='left', on=['communityName'])\n",
    "\n",
    "    a = df.groupby(['plate', 'area_part']).apply(lambda x: x['totalRoom'].tolist()).reset_index()\n",
    "    a['plate_mean_totalRoom'] = [np.mean(i) for i in a[0]]\n",
    "    a['plate_median_totalRoom'] = [np.median(i) for i in a[0]]\n",
    "    a = a.drop(0, axis=1)\n",
    "    df = df.merge(a, how='left', on=['plate', 'area_part'])\n",
    "\n",
    "    a = df.groupby(['plate', 'area_part']).apply(lambda x: x['room'].tolist()).reset_index()\n",
    "    a['plate_mean_room'] = [np.mean(i) for i in a[0]]\n",
    "    a['plate_median_room'] = [np.median(i) for i in a[0]]\n",
    "    a = a.drop(0, axis=1)\n",
    "    df = df.merge(a, how='left', on=['plate', 'area_part'])\n",
    "    \n",
    "    # categorical_feats\n",
    "    categorical_feats = [ 'region', 'plate']\n",
    "    return df, categorical_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getData(feature):\n",
    "    \"\"\"\n",
    "    获取数据\n",
    "    \"\"\"\n",
    "    train = pd.read_csv('train_data.csv')\n",
    "    test = pd.read_csv('test_a.csv')\n",
    "    \n",
    "    train = parseData(train)\n",
    "    test = parseData(test)\n",
    "    train, test = washData(train, test)\n",
    "    \n",
    "    train, col = feature(train)\n",
    "    test, col = feature(test)\n",
    "    \n",
    "    target = train.pop('tradeMoney')\n",
    "    features = train.columns\n",
    "    categorical_feats = col\n",
    "    \n",
    "    return train, test, target, features, categorical_feats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, test, target, features, categorical_feats = getData(feature)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# feature selection\n",
    "## Filter\n",
    "### 相关系数法\n",
    "### 卡方检验"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import SelectKBest,SelectPercentile\n",
    "from sklearn.feature_selection import chi2\n",
    "\n",
    "X = train_data.drop([\"tradeMoney\"],axis=1)\n",
    "y = train_data[\"tradeMoney\"]\n",
    "\n",
    "# 去掉字符型特征\n",
    "for col in X.columns:\n",
    "    if X[col].dtype.name == \"category\":\n",
    "        X = X.drop([col],axis=1)\n",
    "        \n",
    "X_new = SelectKBest(chi2, k=43).fit(X, y).get_support(indices = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Wrapper\n",
    "### 递归特征消除法(RFE)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embedded\n",
    "### 基于惩罚项的特征选择法\n",
    "### 基于树模型的特征选择法"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
